<!DOCTYPE html><html lang="en" data-theme="light"><head><meta charset="UTF-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no"><title>初识大数据(一) | Lorem MoonのStory</title><meta name="keywords" content="Hadoop,BigData,Spark,Hive"><meta name="author" content="Lorem Moon"><meta name="copyright" content="Lorem Moon"><meta name="format-detection" content="telephone=no"><meta name="theme-color" content="#ffffff"><meta name="description" content="大数据环境的Apache方式搭建">
<meta property="og:type" content="article">
<meta property="og:title" content="初识大数据(一)">
<meta property="og:url" content="https://hello-github-ui.github.io/posts/1553/index.html">
<meta property="og:site_name" content="Lorem MoonのStory">
<meta property="og:description" content="大数据环境的Apache方式搭建">
<meta property="og:locale" content="en_US">
<meta property="og:image" content="https://cdn.jsdelivr.net/gh/jerryc127/CDN@latest/cover/default_bg3.png">
<meta property="article:published_time" content="2022-03-01T07:53:43.000Z">
<meta property="article:modified_time" content="2022-03-01T08:24:12.000Z">
<meta property="article:author" content="Lorem Moon">
<meta property="article:tag" content="Hadoop">
<meta property="article:tag" content="BigData">
<meta property="article:tag" content="Spark">
<meta property="article:tag" content="Hive">
<meta name="twitter:card" content="summary">
<meta name="twitter:image" content="https://cdn.jsdelivr.net/gh/jerryc127/CDN@latest/cover/default_bg3.png"><link rel="shortcut icon" href="/img/favicon.png"><link rel="canonical" href="https://hello-github-ui.github.io/posts/1553/"><link rel="preconnect" href="//cdn.jsdelivr.net"/><link rel="preconnect" href="//busuanzi.ibruce.info"/><link rel="stylesheet" href="/css/index.css"><link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@fortawesome/fontawesome-free/css/all.min.css" media="print" onload="this.media='all'"><link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@fancyapps/ui/dist/fancybox.css" media="print" onload="this.media='all'"><script>const GLOBAL_CONFIG = { 
  root: '/',
  algolia: {"appId":"D5QZVMNT9K","apiKey":"92de0b03da46fa778c95817524081160","indexName":"hexo-blog","hits":{"per_page":100},"languages":{"input_placeholder":"Search for Posts","hits_empty":"We didn't find any results for the search: ${query}.","hits_stats":"${hits} results found in ${time} ms"}},
  localSearch: undefined,
  translate: undefined,
  noticeOutdate: {"limitDay":365,"position":"top","messagePrev":"It has been","messageNext":"days since the last update, the content of the article may be outdated."},
  highlight: {"plugin":"highlighjs","highlightCopy":true,"highlightLang":true,"highlightHeightLimit":200},
  copy: {
    success: 'Copy successfully',
    error: 'Copy error',
    noSupport: 'The browser does not support'
  },
  relativeDate: {
    homepage: false,
    post: false
  },
  runtime: 'days',
  date_suffix: {
    just: 'Just',
    min: 'minutes ago',
    hour: 'hours ago',
    day: 'days ago',
    month: 'months ago'
  },
  copyright: undefined,
  lightbox: 'fancybox',
  Snackbar: undefined,
  source: {
    justifiedGallery: {
      js: 'https://cdn.jsdelivr.net/npm/flickr-justified-gallery@2/dist/fjGallery.min.js',
      css: 'https://cdn.jsdelivr.net/npm/flickr-justified-gallery@2/dist/fjGallery.min.css'
    }
  },
  isPhotoFigcaption: false,
  islazyload: false,
  isAnchor: false
}</script><script id="config-diff">var GLOBAL_CONFIG_SITE = {
  title: '初识大数据(一)',
  isPost: true,
  isHome: false,
  isHighlightShrink: true,
  isToc: true,
  postUpdate: '2022-03-01 16:24:12'
}</script><noscript><style type="text/css">
  #nav {
    opacity: 1
  }
  .justified-gallery img {
    opacity: 1
  }

  #recent-posts time,
  #post-meta time {
    display: inline !important
  }
</style></noscript><script>(win=>{
    win.saveToLocal = {
      set: function setWithExpiry(key, value, ttl) {
        if (ttl === 0) return
        const now = new Date()
        const expiryDay = ttl * 86400000
        const item = {
          value: value,
          expiry: now.getTime() + expiryDay,
        }
        localStorage.setItem(key, JSON.stringify(item))
      },

      get: function getWithExpiry(key) {
        const itemStr = localStorage.getItem(key)

        if (!itemStr) {
          return undefined
        }
        const item = JSON.parse(itemStr)
        const now = new Date()

        if (now.getTime() > item.expiry) {
          localStorage.removeItem(key)
          return undefined
        }
        return item.value
      }
    }
  
    win.getScript = url => new Promise((resolve, reject) => {
      const script = document.createElement('script')
      script.src = url
      script.async = true
      script.onerror = reject
      script.onload = script.onreadystatechange = function() {
        const loadState = this.readyState
        if (loadState && loadState !== 'loaded' && loadState !== 'complete') return
        script.onload = script.onreadystatechange = null
        resolve()
      }
      document.head.appendChild(script)
    })
  
      win.activateDarkMode = function () {
        document.documentElement.setAttribute('data-theme', 'dark')
        if (document.querySelector('meta[name="theme-color"]') !== null) {
          document.querySelector('meta[name="theme-color"]').setAttribute('content', '#0d0d0d')
        }
      }
      win.activateLightMode = function () {
        document.documentElement.setAttribute('data-theme', 'light')
        if (document.querySelector('meta[name="theme-color"]') !== null) {
          document.querySelector('meta[name="theme-color"]').setAttribute('content', '#ffffff')
        }
      }
      const t = saveToLocal.get('theme')
    
          if (t === 'dark') activateDarkMode()
          else if (t === 'light') activateLightMode()
        
      const asideStatus = saveToLocal.get('aside-status')
      if (asideStatus !== undefined) {
        if (asideStatus === 'hide') {
          document.documentElement.classList.add('hide-aside')
        } else {
          document.documentElement.classList.remove('hide-aside')
        }
      }
    
    const detectApple = () => {
      if(/iPad|iPhone|iPod|Macintosh/.test(navigator.userAgent)){
        document.documentElement.classList.add('apple')
      }
    }
    detectApple()
    })(window)</script><meta name="generator" content="Hexo 5.4.0"></head><body><div id="web_bg"></div><div id="sidebar"><div id="menu-mask"></div><div id="sidebar-menus"><div class="avatar-img is-center"><img src="https://pic.imgdb.cn/item/6159d89b2ab3f51d91b0e8d0.jpg" onerror="onerror=null;src='/img/friend_404.gif'" alt="avatar"/></div><div class="site-data is-center"><div class="data-item"><a href="/archives/"><div class="headline">Articles</div><div class="length-num">60</div></a></div><div class="data-item"><a href="/tags/"><div class="headline">Tags</div><div class="length-num">46</div></a></div><div class="data-item"><a href="/categories/"><div class="headline">Categories</div><div class="length-num">5</div></a></div></div><hr/><div class="menus_items"><div class="menus_item"><a class="site-page" href="/"><i class="fa-fw fas fa-home"></i><span> Home</span></a></div><div class="menus_item"><a class="site-page" href="/archives/"><i class="fa-fw fas fa-archive"></i><span> Archives</span></a></div><div class="menus_item"><a class="site-page" href="/tags/"><i class="fa-fw fas fa-tags"></i><span> Tags</span></a></div><div class="menus_item"><a class="site-page" href="/categories/"><i class="fa-fw fas fa-folder-open"></i><span> Categories</span></a></div><div class="menus_item"><a class="site-page group" href="javascript:void(0);"><i class="fa-fw fas fa-list"></i><span> List</span><i class="fas fa-chevron-down"></i></a><ul class="menus_item_child"><li><a class="site-page child" href="/musics/"><i class="fa-fw fas fa-music"></i><span> Music</span></a></li><li><a class="site-page child" href="/movies/"><i class="fa-fw fas fa-video"></i><span> Movie</span></a></li></ul></div><div class="menus_item"><a class="site-page" href="/link/"><i class="fa-fw fas fa-link"></i><span> Link</span></a></div><div class="menus_item"><a class="site-page" href="/about/"><i class="fa-fw fas fa-heart"></i><span> About</span></a></div></div></div></div><div class="post" id="body-wrap"><header class="post-bg" id="page-header" style="background-image: url('https://cdn.jsdelivr.net/gh/jerryc127/CDN@latest/cover/default_bg3.png')"><nav id="nav"><span id="blog_name"><a id="site-name" href="/">Lorem MoonのStory</a></span><div id="menus"><div id="search-button"><a class="site-page social-icon search"><i class="fas fa-search fa-fw"></i><span> Search</span></a></div><div class="menus_items"><div class="menus_item"><a class="site-page" href="/"><i class="fa-fw fas fa-home"></i><span> Home</span></a></div><div class="menus_item"><a class="site-page" href="/archives/"><i class="fa-fw fas fa-archive"></i><span> Archives</span></a></div><div class="menus_item"><a class="site-page" href="/tags/"><i class="fa-fw fas fa-tags"></i><span> Tags</span></a></div><div class="menus_item"><a class="site-page" href="/categories/"><i class="fa-fw fas fa-folder-open"></i><span> Categories</span></a></div><div class="menus_item"><a class="site-page group" href="javascript:void(0);"><i class="fa-fw fas fa-list"></i><span> List</span><i class="fas fa-chevron-down"></i></a><ul class="menus_item_child"><li><a class="site-page child" href="/musics/"><i class="fa-fw fas fa-music"></i><span> Music</span></a></li><li><a class="site-page child" href="/movies/"><i class="fa-fw fas fa-video"></i><span> Movie</span></a></li></ul></div><div class="menus_item"><a class="site-page" href="/link/"><i class="fa-fw fas fa-link"></i><span> Link</span></a></div><div class="menus_item"><a class="site-page" href="/about/"><i class="fa-fw fas fa-heart"></i><span> About</span></a></div></div><div id="toggle-menu"><a class="site-page"><i class="fas fa-bars fa-fw"></i></a></div></div></nav><div id="post-info"><h1 class="post-title">初识大数据(一)<a class="post-edit-link" href="https://github.com/hello-github-ui/hexo-blog-source/tree/master/source/_posts/初识大数据(一).md" title="Edited on" target="_blank"><i class="fas fa-pencil-alt"></i></a></h1><div id="post-meta"><div class="meta-firstline"><span class="post-meta-date"><i class="far fa-calendar-alt fa-fw post-meta-icon"></i><span class="post-meta-label">Created</span><time class="post-meta-date-created" datetime="2022-03-01T07:53:43.000Z" title="Created 2022-03-01 15:53:43">2022-03-01</time><span class="post-meta-separator">|</span><i class="fas fa-history fa-fw post-meta-icon"></i><span class="post-meta-label">Updated</span><time class="post-meta-date-updated" datetime="2022-03-01T08:24:12.000Z" title="Updated 2022-03-01 16:24:12">2022-03-01</time></span><span class="post-meta-categories"><span class="post-meta-separator">|</span><i class="fas fa-inbox fa-fw post-meta-icon"></i><a class="post-meta-categories" href="/categories/%E8%AE%B0%E4%BA%8B%E6%9C%AC/">记事本</a></span></div><div class="meta-secondline"><span class="post-meta-separator">|</span><span class="post-meta-wordcount"><i class="far fa-file-word fa-fw post-meta-icon"></i><span class="post-meta-label">Word count:</span><span class="word-count">4.6k</span><span class="post-meta-separator">|</span><i class="far fa-clock fa-fw post-meta-icon"></i><span class="post-meta-label">Reading time:</span><span>21min</span></span><span class="post-meta-separator">|</span><span class="post-meta-pv-cv" id="" data-flag-title="初识大数据(一)"><i class="far fa-eye fa-fw post-meta-icon"></i><span class="post-meta-label">Post View:</span><span id="busuanzi_value_page_pv"></span></span></div></div></div></header><main class="layout" id="content-inner"><div id="post"><article class="post-content" id="article-container"><h1 id="大数据基础"><a href="#大数据基础" class="headerlink" title="大数据基础"></a>大数据基础</h1><blockquote>
<p>Hadoop 三大发行版本：Apache、Cloudera、Hortonworks。这里以普通的 Apache Hadoop 方式搭建大数据环境（后面会考虑使用更贴合实际的 CDH 版本的 Hadoop集群）</p>
</blockquote>
<h2 id="Hadoop-Yarn集群的搭建"><a href="#Hadoop-Yarn集群的搭建" class="headerlink" title="Hadoop Yarn集群的搭建"></a>Hadoop Yarn集群的搭建</h2><h3 id="虚拟机的准备"><a href="#虚拟机的准备" class="headerlink" title="虚拟机的准备"></a>虚拟机的准备</h3><blockquote>
<p>先准备三台centos7虚拟机，具体方式不做演示</p>
</blockquote>
<h4 id="修改-IP"><a href="#修改-IP" class="headerlink" title="修改 IP"></a>修改 IP</h4><ol>
<li>第一台虚拟机修改为如下内容：</li>
</ol>
<figure class="highlight shell"><table><tr><td class="code"><pre><span class="line"><span class="meta">#</span><span class="bash">  vim /etc/sysconfig/network-scripts/ifcfgens33</span></span><br><span class="line">TYPE=Ethernet</span><br><span class="line">PROXY_METHOD=none</span><br><span class="line">BROWSER_ONLY=no</span><br><span class="line">BOOTPROTO=static</span><br><span class="line">DEFROUTE=yes</span><br><span class="line">IPV4_FAILURE_FATAL=no</span><br><span class="line">IPV6INIT=yes</span><br><span class="line">IPV6_AUTOCONF=yes</span><br><span class="line">IPV6_DEFROUTE=yes</span><br><span class="line">IPV6_FAILURE_FATAL=no</span><br><span class="line">IPV6_ADDR_GEN_MODE=stable-privacy</span><br><span class="line">NAME=ens33</span><br><span class="line">UUID=29b35945-26d7-4f4f-9924-ff97b9bf0638</span><br><span class="line">DEVICE=ens33</span><br><span class="line">ONBOOT=yes</span><br><span class="line">IPADDR=10.144.144.102</span><br><span class="line">PREFIX=24</span><br><span class="line">GATEWAY=10.144.144.2</span><br><span class="line">DNS1=10.144.144.2</span><br></pre></td></tr></table></figure>

<ol start="2">
<li>第二台虚拟机修改为如下内容：</li>
</ol>
<figure class="highlight shell"><table><tr><td class="code"><pre><span class="line">BOOTPROTO=&quot;static&quot;</span><br><span class="line">DEFROUTE=&quot;yes&quot;</span><br><span class="line">IPV4_FAILURE_FATAL=&quot;no&quot;</span><br><span class="line">IPV6INIT=&quot;yes&quot;</span><br><span class="line">IPV6_AUTOCONF=&quot;yes&quot;</span><br><span class="line">IPV6_DEFROUTE=&quot;yes&quot;</span><br><span class="line">IPV6_FAILURE_FATAL=&quot;no&quot;</span><br><span class="line">IPV6_ADDR_GEN_MODE=&quot;stable-privacy&quot;</span><br><span class="line">NAME=&quot;ens33&quot;</span><br><span class="line">UUID=&quot;07ad7c52-3d68-4e89-a699-8be6a0bd92d0&quot;</span><br><span class="line">DEVICE=&quot;ens33&quot;</span><br><span class="line">ONBOOT=&quot;yes&quot;</span><br><span class="line">IPADDR=&quot;10.144.144.103&quot;</span><br><span class="line">PREFIX=&quot;24&quot;</span><br><span class="line">GATEWAY=&quot;10.144.144.2&quot;</span><br><span class="line">DNS1=&quot;10.144.144.2&quot;</span><br></pre></td></tr></table></figure>

<ol start="2">
<li>第三台虚拟机修改为如下内容：</li>
</ol>
<figure class="highlight shell"><table><tr><td class="code"><pre><span class="line">BOOTPROTO=&quot;static&quot;</span><br><span class="line">DEFROUTE=&quot;yes&quot;</span><br><span class="line">IPV4_FAILURE_FATAL=&quot;no&quot;</span><br><span class="line">IPV6INIT=&quot;yes&quot;</span><br><span class="line">IPV6_AUTOCONF=&quot;yes&quot;</span><br><span class="line">IPV6_DEFROUTE=&quot;yes&quot;</span><br><span class="line">IPV6_FAILURE_FATAL=&quot;no&quot;</span><br><span class="line">IPV6_ADDR_GEN_MODE=&quot;stable-privacy&quot;</span><br><span class="line">NAME=&quot;ens33&quot;</span><br><span class="line">UUID=&quot;07ad7c52-3d68-4e89-a699-8be6a0bd92d0&quot;</span><br><span class="line">DEVICE=&quot;ens33&quot;</span><br><span class="line">ONBOOT=&quot;yes&quot;</span><br><span class="line">IPADDR=&quot;10.144.144.104&quot;</span><br><span class="line">PREFIX=&quot;24&quot;</span><br><span class="line">GATEWAY=&quot;10.144.144.2&quot;</span><br><span class="line">DNS1=&quot;10.144.144.2&quot;</span><br></pre></td></tr></table></figure>

<h4 id="修改主机名"><a href="#修改主机名" class="headerlink" title="修改主机名"></a>修改主机名</h4><ol>
<li>第一台虚拟机修改为如下内容：</li>
</ol>
<figure class="highlight shell"><table><tr><td class="code"><pre><span class="line"><span class="meta">#</span><span class="bash"> vim /etc/hostname</span></span><br><span class="line">hadoop102</span><br></pre></td></tr></table></figure>

<ol>
<li>第二台虚拟机修改为如下内容：</li>
</ol>
<figure class="highlight shell"><table><tr><td class="code"><pre><span class="line"><span class="meta">#</span><span class="bash"> vim /etc/hostname</span></span><br><span class="line">hadoop103</span><br></pre></td></tr></table></figure>
<ol start="2">
<li>第三台虚拟机修改为如下内容：</li>
</ol>
<figure class="highlight shell"><table><tr><td class="code"><pre><span class="line"><span class="meta">#</span><span class="bash"> vim /etc/hostname</span></span><br><span class="line">hadoop104</span><br></pre></td></tr></table></figure>

<h4 id="配置-Linux-克隆机主机名称映射-hosts-文件，打开-etc-hosts"><a href="#配置-Linux-克隆机主机名称映射-hosts-文件，打开-etc-hosts" class="headerlink" title="配置 Linux 克隆机主机名称映射 hosts 文件，打开/etc/hosts"></a>配置 Linux 克隆机主机名称映射 hosts 文件，打开/etc/hosts</h4><figure class="highlight shell"><table><tr><td class="code"><pre><span class="line"><span class="meta">#</span><span class="bash"> vim /etc/hosts 三台主机都如此修改即可</span></span><br><span class="line">10.144.144.144 my_centos</span><br><span class="line">10.144.144.102 hadoop102</span><br><span class="line">10.144.144.103 hadoop103</span><br><span class="line">10.144.144.104 hadoop104</span><br><span class="line">10.144.144.254 my_portable_centos</span><br></pre></td></tr></table></figure>

<h4 id="Windows-host文件配置"><a href="#Windows-host文件配置" class="headerlink" title="Windows host文件配置"></a>Windows host文件配置</h4><figure class="highlight shell"><table><tr><td class="code"><pre><span class="line"><span class="meta">#</span><span class="bash"> windows/system32/drivers/etc/hosts</span></span><br><span class="line">10.144.144.144 my_centos</span><br><span class="line">10.144.144.102 hadoop102</span><br><span class="line">10.144.144.103 hadoop103</span><br><span class="line">10.144.144.104 hadoop104</span><br><span class="line">10.144.144.254 my_portable_centos</span><br></pre></td></tr></table></figure>

<h4 id="卸载自带的-jdk"><a href="#卸载自带的-jdk" class="headerlink" title="卸载自带的 jdk"></a>卸载自带的 jdk</h4><figure class="highlight shell"><table><tr><td class="code"><pre><span class="line">rpm -qa | grep -i java | xargs -n1 rpm -e --nodeps</span><br></pre></td></tr></table></figure>

<p>➢ rpm -qa：查询所安装的所有 rpm 软件包</p>
<p>➢ grep -i：忽略大小写</p>
<p>➢ xargs -n1：表示每次只传递一个参数</p>
<p>➢ rpm -e –nodeps：强制卸载软件</p>
<h4 id="关闭防火墙并禁用开机自启动防火墙"><a href="#关闭防火墙并禁用开机自启动防火墙" class="headerlink" title="关闭防火墙并禁用开机自启动防火墙"></a>关闭防火墙并禁用开机自启动防火墙</h4><figure class="highlight shell"><table><tr><td class="code"><pre><span class="line"><span class="meta">#</span><span class="bash"> hadoop102 hadoop103 hadoop104 都需要如下配置</span></span><br><span class="line">systemctl stop firewalld</span><br><span class="line">systemctl disable firewalld.service</span><br></pre></td></tr></table></figure>

<h4 id="创建普通用户"><a href="#创建普通用户" class="headerlink" title="创建普通用户"></a>创建普通用户</h4><figure class="highlight shell"><table><tr><td class="code"><pre><span class="line"><span class="meta">#</span><span class="bash"> 使用 root 账户操作</span></span><br><span class="line"> useradd admin</span><br><span class="line"> passwd admin # 回车后，提示输入新密码，建议123456</span><br></pre></td></tr></table></figure>

<h4 id="配置-admin-用户具有-root-权限，方便后期加-sudo-执行-root-权限的命令"><a href="#配置-admin-用户具有-root-权限，方便后期加-sudo-执行-root-权限的命令" class="headerlink" title="配置 admin 用户具有 root 权限，方便后期加 sudo 执行 root 权限的命令"></a>配置 admin 用户具有 root 权限，方便后期加 sudo 执行 root 权限的命令</h4><p>修改/etc/sudoers 文件，在%wheel 这行下面添加一行，如下所示：</p>
<figure class="highlight shell"><table><tr><td class="code"><pre><span class="line"><span class="meta">#</span><span class="bash"> vim /etc/sudoers</span></span><br><span class="line"><span class="meta">#</span><span class="bash"><span class="comment"># Allow root to run any commands anywhere</span></span></span><br><span class="line">root ALL=(ALL) ALL</span><br><span class="line"><span class="meta">#</span><span class="bash"><span class="comment"># Allows people in group wheel to run all commands</span></span></span><br><span class="line"><span class="meta">%</span><span class="bash">wheel ALL=(ALL) ALL</span></span><br><span class="line">admin ALL=(ALL) NOPASSWD:ALL # 注意，位置一定要放到 wheel 下面一行</span><br></pre></td></tr></table></figure>

<blockquote>
<p> 注意：admin 这一行不要直接放到 root 行下面，因为所有用户都属于 wheel 组，你先 配置了 admin 具有免密功能，但是程序执行到%wheel 行时，该功能又被覆盖回需要 密码。所以 admin 要放到%wheel 这行下面。</p>
</blockquote>
<h4 id="在-opt-目录下创建文件夹，并修改所属主和所属组"><a href="#在-opt-目录下创建文件夹，并修改所属主和所属组" class="headerlink" title="在/opt 目录下创建文件夹，并修改所属主和所属组"></a>在/opt 目录下创建文件夹，并修改所属主和所属组</h4><p>（1）在/opt 目录下创建 module、software 文件夹 【三台主机都需要如下操作，如果你是采用的clone模式另外两台主机，只需要Hadoop102操作即可】</p>
<figure class="highlight shell"><table><tr><td class="code"><pre><span class="line">[root@hadoop102 ~]# mkdir /opt/module</span><br><span class="line">[root@hadoop102 ~]# mkdir /opt/software</span><br></pre></td></tr></table></figure>

<p>（2）修改 module、software 文件夹的所有者和所属组均为 admin 用户</p>
<figure class="highlight shell"><table><tr><td class="code"><pre><span class="line">[root@hadoop102 ~]# chown admin:admin /opt/module</span><br><span class="line">[root@hadoop102 ~]# chown admin:admin /opt/software</span><br></pre></td></tr></table></figure>

<blockquote>
<p>三台主机都编辑完成后，reboot 。</p>
</blockquote>
<h4 id="Hadoop102上安装jdk"><a href="#Hadoop102上安装jdk" class="headerlink" title="Hadoop102上安装jdk"></a>Hadoop102上安装jdk</h4><ul>
<li><p>用 XShell 传输工具将 JDK 导入到 opt 目录下面的 software 文件夹下面</p>
</li>
<li><p>解压 JDK 到/opt/module 目录下 <code>tar -zxvf jdkjdk-8u311-linux-x64.tar.gz -C /opt/module/</code></p>
</li>
<li><p>配置 JDK 环境变量</p>
<p>（1）新建/etc/profile.d/my_env.sh 文件</p>
<figure class="highlight shell"><table><tr><td class="code"><pre><span class="line">[admin@hadoop102 ~]$ sudo vim /etc/profile.d/my_env.sh</span><br></pre></td></tr></table></figure>

<p>添加如下内容</p>
<figure class="highlight shell"><table><tr><td class="code"><pre><span class="line"><span class="meta">#</span><span class="bash">JAVA_HOME</span></span><br><span class="line">export JAVA_HOME=/opt/module/jdk1.8.0_311</span><br><span class="line">export PATH=$PATH:$JAVA_HOME/bin</span><br></pre></td></tr></table></figure>

<p>保存, source 一下/etc/profile 文件，让新的环境变量 PATH 生效即可</p>
</li>
</ul>
<h4 id="编写集群分发脚本-xsync"><a href="#编写集群分发脚本-xsync" class="headerlink" title="编写集群分发脚本 xsync"></a>编写集群分发脚本 xsync</h4><blockquote>
<p>（1）需求：循环复制文件到所有节点的相同目录下</p>
</blockquote>
<figure class="highlight shell"><table><tr><td class="code"><pre><span class="line"><span class="meta">#</span><span class="bash"> 在/home/admin/bin 目录下创建 xsync 文件</span></span><br><span class="line"><span class="meta">#</span><span class="bash"> 在该文件中编写如下代码</span></span><br><span class="line"><span class="meta"></span></span><br><span class="line"><span class="meta">#</span><span class="bash">!/bin/bash</span></span><br><span class="line"><span class="meta">#</span><span class="bash">1. 判断参数个数</span></span><br><span class="line">if [ $# -lt 1 ]; then</span><br><span class="line">	echo Not Enough Arguement!</span><br><span class="line">	exit</span><br><span class="line">fi</span><br><span class="line"><span class="meta">#</span><span class="bash">2. 遍历集群所有机器</span></span><br><span class="line">for host in hadoop102 hadoop103 hadoop104; do</span><br><span class="line">	echo ==================== $host ====================</span><br><span class="line"><span class="meta">	#</span><span class="bash">3. 遍历所有目录，挨个发送</span></span><br><span class="line">	for file in $@; do</span><br><span class="line"><span class="meta">		#</span><span class="bash">4. 判断文件是否存在</span></span><br><span class="line">		if [ -e $file ]; then</span><br><span class="line"><span class="meta">			#</span><span class="bash">5. 获取父目录</span></span><br><span class="line">			pdir=$(</span><br><span class="line">				cd -P $(dirname $file)</span><br><span class="line">				pwd</span><br><span class="line">			)</span><br><span class="line"><span class="meta">			#</span><span class="bash">6. 获取当前文件的名称</span></span><br><span class="line">			fname=$(basename $file)</span><br><span class="line">			ssh $host &quot;mkdir -p $pdir&quot;</span><br><span class="line">			rsync -av $pdir/$fname $host:$pdir</span><br><span class="line">		else</span><br><span class="line">			echo $file does not exists!</span><br><span class="line">		fi</span><br><span class="line">	done</span><br><span class="line">done</span><br></pre></td></tr></table></figure>

<blockquote>
<p>（2）修改脚本 xsync 具有执行权限</p>
<figure class="highlight shell"><table><tr><td class="code"><pre><span class="line">[admin@hadoop102 bin]$ chmod +x xsync</span><br></pre></td></tr></table></figure>

<p>（3）测试脚本</p>
<figure class="highlight shell"><table><tr><td class="code"><pre><span class="line">[admin@hadoop102 ~]$ xsync /home/admin/bin</span><br></pre></td></tr></table></figure>

<p>（4）将脚本复制到/bin 中，以便全局调用</p>
<figure class="highlight shell"><table><tr><td class="code"><pre><span class="line">[admin@hadoop102 bin]$ sudo cp xsync /bin/</span><br></pre></td></tr></table></figure>

<p>（5）同步环境变量配置（root 所有者）</p>
<figure class="highlight shell"><table><tr><td class="code"><pre><span class="line">[admin@hadoop102 ~]$ sudo ./bin/xsync /etc/profile.d/my_env.sh</span><br></pre></td></tr></table></figure>

<p>注意：如果用了 sudo，那么 xsync 一定要给它的路径补全。</p>
<p>（6）让环境变量生效</p>
<figure class="highlight shell"><table><tr><td class="code"><pre><span class="line">[admin@hadoop102 bin]$ source /etc/profile</span><br></pre></td></tr></table></figure>
</blockquote>
<h4 id="SSH-无密登录配置"><a href="#SSH-无密登录配置" class="headerlink" title="SSH 无密登录配置"></a>SSH 无密登录配置</h4><p>需求：ssh 另一台电脑的ip地址</p>
<ul>
<li>生成公钥和私钥</li>
</ul>
<figure class="highlight shell"><table><tr><td class="code"><pre><span class="line">[root@hadoop102 .ssh]$ pwd</span><br><span class="line">/root/.ssh</span><br><span class="line">[root@hadoop102 .ssh]$ ssh-keygen -t rsa</span><br></pre></td></tr></table></figure>

<p>然后敲（三个回车），就会生成两个文件 id_rsa（私钥）、id_rsa.pub（公钥）</p>
<ul>
<li>将公钥拷贝到要免密登录的目标机器上</li>
</ul>
<figure class="highlight shell"><table><tr><td class="code"><pre><span class="line">[root@hadoop102 .ssh]$ ssh-copy-id hadoop102</span><br><span class="line">[root@hadoop102 .ssh]$ ssh-copy-id hadoop103</span><br><span class="line">[root@hadoop102 .ssh]$ ssh-copy-id hadoop104</span><br></pre></td></tr></table></figure>

<ul>
<li>登录 hadoop103，104 ，在其上也执行一遍如上操作</li>
</ul>
<figure class="highlight shell"><table><tr><td class="code"><pre><span class="line"><span class="meta">#</span><span class="bash"> 登录 hadoop103</span></span><br><span class="line">[root@hadoop103 .ssh]$ ssh-copy-id hadoop102</span><br><span class="line">[root@hadoop103 .ssh]$ ssh-copy-id hadoop103</span><br><span class="line">[root@hadoop103 .ssh]$ ssh-copy-id hadoop104</span><br><span class="line"><span class="meta">#</span><span class="bash"> 登录 hadoop104</span></span><br><span class="line">[root@hadoop104 .ssh]$ ssh-copy-id hadoop102</span><br><span class="line">[root@hadoop104 .ssh]$ ssh-copy-id hadoop103</span><br><span class="line">[root@hadoop104 .ssh]$ ssh-copy-id hadoop104</span><br></pre></td></tr></table></figure>

<ul>
<li>切换成 admin 用户，然后重复一遍上面的操作即可【注意：不需要重新生成 id_rsa 了】</li>
</ul>
<h4 id="同步-jdk-到-Hadoop103，Hadoop104"><a href="#同步-jdk-到-Hadoop103，Hadoop104" class="headerlink" title="同步 jdk 到 Hadoop103，Hadoop104"></a>同步 jdk 到 Hadoop103，Hadoop104</h4><figure class="highlight shell"><table><tr><td class="code"><pre><span class="line">[admin@hadoop102 module]$ xsync jdk1.8.0_311/</span><br><span class="line"><span class="meta">#</span><span class="bash"> 同步 my_env.sh</span></span><br><span class="line">[admin@hadoop102 module]$ xsync /etc/profile.d/my_env.sh</span><br></pre></td></tr></table></figure>



<h3 id="Hadoop-安装"><a href="#Hadoop-安装" class="headerlink" title="Hadoop 安装"></a>Hadoop 安装</h3><p>Hadoop 下载地址：<a target="_blank" rel="noopener" href="https://archive.apache.org/dist/hadoop/common/hadoop-3.1.3/">https://archive.apache.org/dist/hadoop/common/hadoop-3.1.3/</a></p>
<p>1）用 XShell 文件传输工具将 hadoop-3.1.3.tar.gz 导入到 opt 目录下面的 software 文件夹下 面</p>
<p>2）进入到 Hadoop 安装包路径下 [atguigu@hadoop102 ~]$ cd /opt/software/</p>
<p>3）解压安装文件到/opt/module 下面</p>
<figure class="highlight shell"><table><tr><td class="code"><pre><span class="line">[admin@hadoop102 software]$ tar -zxvf hadoop-3.1.3.tar.gz -C /opt/module/</span><br></pre></td></tr></table></figure>

<p>4）查看是否解压成功</p>
<figure class="highlight shell"><table><tr><td class="code"><pre><span class="line">[admin@hadoop102 software]$ ls /opt/module/ hadoop-3.1.3</span><br></pre></td></tr></table></figure>

<p> 5）将 Hadoop 添加到环境变量</p>
<p>​    （1）获取 Hadoop 安装路径 <code>[admin@hadoop102 hadoop-3.1.3]$ pwd /opt/module/hadoop-3.1.3 </code></p>
<p>​    （2）打开/etc/profile.d/my_env.sh 文件 <code>[admin@hadoop102 hadoop-3.1.3]$ sudo vim /etc/profile.d/my_env.sh</code></p>
<figure class="highlight shell"><table><tr><td class="code"><pre><span class="line"><span class="meta">#</span><span class="bash">HADOOP_HOME</span></span><br><span class="line">export HADOOP_HOME=/opt/module/hadoop-3.1.3</span><br><span class="line">export PATH=$PATH:$HADOOP_HOME/bin</span><br><span class="line">export PATH=$PATH:$HADOOP_HOME/sbin</span><br></pre></td></tr></table></figure>

<h3 id="Hadoop-Yarn集群配置"><a href="#Hadoop-Yarn集群配置" class="headerlink" title="Hadoop Yarn集群配置"></a>Hadoop Yarn集群配置</h3><h4 id="配置-core-site-xml"><a href="#配置-core-site-xml" class="headerlink" title="配置 core-site.xml"></a>配置 core-site.xml</h4><p>核心配置文件 vim  $HADOOP_HOME/etc/hadoop/core-site.xml</p>
<figure class="highlight xml"><table><tr><td class="code"><pre><span class="line"><span class="meta">&lt;?xml version=&quot;1.0&quot; encoding=&quot;UTF-8&quot;?&gt;</span></span><br><span class="line"><span class="meta">&lt;?xml-stylesheet type=&quot;text/xsl&quot; href=&quot;configuration.xsl&quot;?&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">configuration</span>&gt;</span></span><br><span class="line"> <span class="comment">&lt;!-- 指定 NameNode 的地址 --&gt;</span></span><br><span class="line"> <span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line"> <span class="tag">&lt;<span class="name">name</span>&gt;</span>fs.defaultFS<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line"> <span class="tag">&lt;<span class="name">value</span>&gt;</span>hdfs://hadoop102:8020<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line"> <span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line"> <span class="comment">&lt;!-- 指定 hadoop 数据的存储目录 --&gt;</span></span><br><span class="line"> <span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line"> <span class="tag">&lt;<span class="name">name</span>&gt;</span>hadoop.tmp.dir<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line"> <span class="tag">&lt;<span class="name">value</span>&gt;</span>/opt/module/hadoop-3.1.3/data<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line"> <span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line"> <span class="comment">&lt;!-- 配置 HDFS 网页登录使用的静态用户为 admin，注意后期 hive 的 beeline 登录用户名也是这个 admin --&gt;</span></span><br><span class="line"> <span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line"> <span class="tag">&lt;<span class="name">name</span>&gt;</span>hadoop.http.staticuser.user<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line"> <span class="tag">&lt;<span class="name">value</span>&gt;</span>admin<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line"> <span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">configuration</span>&gt;</span></span><br></pre></td></tr></table></figure>

<h4 id="配置-hdfs-site-xml"><a href="#配置-hdfs-site-xml" class="headerlink" title="配置 hdfs-site.xml"></a>配置 hdfs-site.xml</h4><p>HDFS 配置文件 vim $HADOOP_HOME/etc/hadoop/hdfs-site.xml</p>
<figure class="highlight xml"><table><tr><td class="code"><pre><span class="line"><span class="meta">&lt;?xml version=&quot;1.0&quot; encoding=&quot;UTF-8&quot;?&gt;</span></span><br><span class="line"><span class="meta">&lt;?xml-stylesheet type=&quot;text/xsl&quot; href=&quot;configuration.xsl&quot;?&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">configuration</span>&gt;</span></span><br><span class="line"><span class="comment">&lt;!-- nn web 端访问地址--&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line"> <span class="tag">&lt;<span class="name">name</span>&gt;</span>dfs.namenode.http-address<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line"> <span class="tag">&lt;<span class="name">value</span>&gt;</span>hadoop102:9870<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line"> <span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line"><span class="comment">&lt;!-- 2nn web 端访问地址--&gt;</span></span><br><span class="line"> <span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line"> <span class="tag">&lt;<span class="name">name</span>&gt;</span>dfs.namenode.secondary.http-address<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line"> <span class="tag">&lt;<span class="name">value</span>&gt;</span>hadoop104:9868<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line"> <span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">configuration</span>&gt;</span></span><br></pre></td></tr></table></figure>

<h4 id="配置-yarn-site-xml"><a href="#配置-yarn-site-xml" class="headerlink" title="配置 yarn-site.xml"></a>配置 yarn-site.xml</h4><p>YARN 配置文件 vim $HADOOP_HOME/etc/hadoop/yarn-site.xml</p>
<figure class="highlight xml"><table><tr><td class="code"><pre><span class="line"><span class="meta">&lt;?xml version=&quot;1.0&quot; encoding=&quot;UTF-8&quot;?&gt;</span></span><br><span class="line"><span class="meta">&lt;?xml-stylesheet type=&quot;text/xsl&quot; href=&quot;configuration.xsl&quot;?&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">configuration</span>&gt;</span></span><br><span class="line"> <span class="comment">&lt;!-- 指定 MR 走 shuffle --&gt;</span></span><br><span class="line"> <span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line"> 	<span class="tag">&lt;<span class="name">name</span>&gt;</span>yarn.nodemanager.aux-services<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line"> 	<span class="tag">&lt;<span class="name">value</span>&gt;</span>mapreduce_shuffle<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line"> <span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line"> <span class="comment">&lt;!-- 指定 ResourceManager 的地址--&gt;</span></span><br><span class="line"> <span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line"> 	<span class="tag">&lt;<span class="name">name</span>&gt;</span>yarn.resourcemanager.hostname<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line"> 	<span class="tag">&lt;<span class="name">value</span>&gt;</span>hadoop103<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line"> <span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line"> <span class="comment">&lt;!-- 环境变量的继承 --&gt;</span></span><br><span class="line"> <span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line"> 	<span class="tag">&lt;<span class="name">name</span>&gt;</span>yarn.nodemanager.env-whitelist<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">	<span class="tag">&lt;<span class="name">value</span>&gt;</span>JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_MAPRED_HOME<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line"> <span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">configuration</span>&gt;</span></span><br></pre></td></tr></table></figure>

<h4 id="配置-mapred-site-xml"><a href="#配置-mapred-site-xml" class="headerlink" title="配置 mapred-site.xml"></a>配置 mapred-site.xml</h4><p>MapReduce 配置文件 vim $HADOOP_HOME/etc/hadoop/mapred-site.xml</p>
<figure class="highlight xml"><table><tr><td class="code"><pre><span class="line"><span class="meta">&lt;?xml version=&quot;1.0&quot; encoding=&quot;UTF-8&quot;?&gt;</span></span><br><span class="line"><span class="meta">&lt;?xml-stylesheet type=&quot;text/xsl&quot; href=&quot;configuration.xsl&quot;?&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">configuration</span>&gt;</span></span><br><span class="line"><span class="comment">&lt;!-- 指定 MapReduce 程序运行在 Yarn 上 --&gt;</span></span><br><span class="line"> <span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line"> 	<span class="tag">&lt;<span class="name">name</span>&gt;</span>mapreduce.framework.name<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line"> 	<span class="tag">&lt;<span class="name">value</span>&gt;</span>yarn<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line"> <span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">configuration</span>&gt;</span></span><br></pre></td></tr></table></figure>

<h4 id="配置-workers"><a href="#配置-workers" class="headerlink" title="配置 workers"></a>配置 workers</h4><p>vim $HADOOP_HOME/etc/hadoop/workers</p>
<p>在该文件中增加如下内容：</p>
<figure class="highlight shell"><table><tr><td class="code"><pre><span class="line">hadoop102</span><br><span class="line">hadoop103</span><br><span class="line">hadoop104</span><br></pre></td></tr></table></figure>

<p><strong>注意：该文件中添加的内容结尾不允许有空格，文件中不允许有空行。</strong></p>
<h4 id="配置历史服务器"><a href="#配置历史服务器" class="headerlink" title="配置历史服务器"></a>配置历史服务器</h4><p>为了查看程序的历史运行情况，需要配置一下历史服务器。具体配置步骤如下：</p>
<h5 id="配置-mapred-site-xml-1"><a href="#配置-mapred-site-xml-1" class="headerlink" title="配置 mapred-site.xml"></a>配置 mapred-site.xml</h5><p>vim $HADOOP_HOME/etc/hadoop/mapred-site.xml</p>
<p>在该文件里面增加如下配置:</p>
<figure class="highlight xml"><table><tr><td class="code"><pre><span class="line"><span class="comment">&lt;!-- 历史服务器端地址 --&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line"> <span class="tag">&lt;<span class="name">name</span>&gt;</span>mapreduce.jobhistory.address<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line"> <span class="tag">&lt;<span class="name">value</span>&gt;</span>hadoop102:10020<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line"><span class="comment">&lt;!-- 历史服务器 web 端地址 --&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line"> <span class="tag">&lt;<span class="name">name</span>&gt;</span>mapreduce.jobhistory.webapp.address<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line"> <span class="tag">&lt;<span class="name">value</span>&gt;</span>hadoop102:19888<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br></pre></td></tr></table></figure>

<p>查看 JobHistory: <a target="_blank" rel="noopener" href="http://hadoop102:19888/jobhistory">http://hadoop102:19888/jobhistory</a></p>
<h4 id="配置日志的聚集"><a href="#配置日志的聚集" class="headerlink" title="配置日志的聚集"></a>配置日志的聚集</h4><p>日志聚集概念：应用运行完成以后，将程序运行日志信息上传到 HDFS 系统上。</p>
<p>注意：开启日志聚集功能，需要重新启动 NodeManager 、ResourceManager 和 HistoryServer。此处建议直接重启 hdfs ,yarn</p>
<h5 id="配置-yarn-site-xml-1"><a href="#配置-yarn-site-xml-1" class="headerlink" title="配置 yarn-site.xml"></a>配置 yarn-site.xml</h5><p>vim $HADOOP_HOME/etc/hadoop/yarn-site.xml</p>
<p>在该文件里面增加如下配置。</p>
<figure class="highlight xml"><table><tr><td class="code"><pre><span class="line"><span class="comment">&lt;!-- 开启日志聚集功能 --&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line"> <span class="tag">&lt;<span class="name">name</span>&gt;</span>yarn.log-aggregation-enable<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line"> <span class="tag">&lt;<span class="name">value</span>&gt;</span>true<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line"><span class="comment">&lt;!-- 设置日志聚集服务器地址 --&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line"> <span class="tag">&lt;<span class="name">name</span>&gt;</span>yarn.log.server.url<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line"> <span class="tag">&lt;<span class="name">value</span>&gt;</span>http://hadoop102:19888/jobhistory/logs<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line"><span class="comment">&lt;!-- 设置日志保留时间为 7 天 --&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line"> <span class="tag">&lt;<span class="name">name</span>&gt;</span>yarn.log-aggregation.retain-seconds<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line"> <span class="tag">&lt;<span class="name">value</span>&gt;</span>604800<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br></pre></td></tr></table></figure>

<h4 id="集群启动-停止方式"><a href="#集群启动-停止方式" class="headerlink" title="集群启动/停止方式"></a>集群启动/停止方式</h4><p>（1）各个模块分开启动/停止（配置 ssh 是前提）常用</p>
<p><code>start-dfs.sh/stop-dfs.sh</code></p>
<p>（2）整体启动/停止 YARN</p>
<p><code>start-yarn.sh/stop-yarn.sh</code></p>
<p>（3）各个服务组件逐一启动/停止</p>
<ul>
<li>分别启动/停止 HDFS 组件 <code>hdfs --daemon start/stop namenode/datanode/secondarynamenode</code></li>
<li>启动/停止 YARN <code>yarn --daemon start/stop resourcemanager/nodemanager</code></li>
</ul>
<h5 id="Hadoop-集群启停脚本（包含-HDFS，Yarn，Historyserver）：myhadoop-sh"><a href="#Hadoop-集群启停脚本（包含-HDFS，Yarn，Historyserver）：myhadoop-sh" class="headerlink" title="Hadoop 集群启停脚本（包含 HDFS，Yarn，Historyserver）：myhadoop.sh"></a>Hadoop 集群启停脚本（包含 HDFS，Yarn，Historyserver）：myhadoop.sh</h5><p>cd /home/admin/bin</p>
<p>vim myhadoop.sh</p>
<p>➢ 输入如下内容</p>
<figure class="highlight shell"><table><tr><td class="code"><pre><span class="line"><span class="meta">#</span><span class="bash">!/bin/bash</span></span><br><span class="line">if [ $# -lt 1 ]; then</span><br><span class="line">	echo &quot;No Args Input...&quot;</span><br><span class="line">	exit</span><br><span class="line">fi</span><br><span class="line">case $1 in</span><br><span class="line">&quot;start&quot;)</span><br><span class="line">	echo &quot; =================== 启动 hadoop 集群 ===================&quot;</span><br><span class="line">	echo &quot; --------------- 启动 hdfs ---------------&quot;</span><br><span class="line">	ssh hadoop102 &quot;/opt/module/hadoop-3.1.3/sbin/start-dfs.sh&quot;</span><br><span class="line">	echo &quot; --------------- 启动 yarn ---------------&quot;</span><br><span class="line">	ssh hadoop103 &quot;/opt/module/hadoop-3.1.3/sbin/start-yarn.sh&quot;</span><br><span class="line">	echo &quot; --------------- 启动 historyserver ---------------&quot;</span><br><span class="line">	ssh hadoop102 &quot;/opt/module/hadoop-3.1.3/bin/mapred --daemon start</span><br><span class="line">historyserver&quot;</span><br><span class="line">	;;</span><br><span class="line">&quot;stop&quot;)</span><br><span class="line">	echo &quot; =================== 关闭 hadoop 集群 ===================&quot;</span><br><span class="line">	echo &quot; --------------- 关闭 historyserver ---------------&quot;</span><br><span class="line">	ssh hadoop102 &quot;/opt/module/hadoop-3.1.3/bin/mapred --daemon stop</span><br><span class="line">historyserver&quot;</span><br><span class="line">	echo &quot; --------------- 关闭 yarn ---------------&quot;</span><br><span class="line">	ssh hadoop103 &quot;/opt/module/hadoop-3.1.3/sbin/stop-yarn.sh&quot;</span><br><span class="line">	echo &quot; --------------- 关闭 hdfs ---------------&quot;</span><br><span class="line">	ssh hadoop102 &quot;/opt/module/hadoop-3.1.3/sbin/stop-dfs.sh&quot;</span><br><span class="line">	;;</span><br><span class="line">*)</span><br><span class="line">	echo &quot;Input Args Error...&quot;</span><br><span class="line">	;;</span><br><span class="line">esac</span><br></pre></td></tr></table></figure>

<p>注意hadoop版本，路径确认一致</p>
<p>赋予脚本执行权限 <code>chmod +x myhadoop.sh</code></p>
<h4 id="查看三台服务器-Java-进程脚本：jpsall"><a href="#查看三台服务器-Java-进程脚本：jpsall" class="headerlink" title="查看三台服务器 Java 进程脚本：jpsall"></a>查看三台服务器 Java 进程脚本：jpsall</h4><p>cd /home/admin/bin</p>
<p>vim jpsall</p>
<p>➢ 输入如下内容</p>
<figure class="highlight shell"><table><tr><td class="code"><pre><span class="line"><span class="meta">#</span><span class="bash">!/bin/bash</span></span><br><span class="line">for host in hadoop102 hadoop103 hadoop104</span><br><span class="line">do</span><br><span class="line"> echo =============== $host ===============</span><br><span class="line"> ssh $host jps</span><br><span class="line">done</span><br></pre></td></tr></table></figure>

<p>赋予脚本执行权限 <code>chmod +x jpsall</code></p>
<h4 id="分发内容到其它主机"><a href="#分发内容到其它主机" class="headerlink" title="分发内容到其它主机"></a>分发内容到其它主机</h4><p>前面已经将 jdk 分发了，此处只需要分发 hadoop my_env.sh 即可</p>
<figure class="highlight shell"><table><tr><td class="code"><pre><span class="line"><span class="meta">#</span><span class="bash"> 分发/home/admin/bin 目录，保证自定义脚本在三台机器上都可以使用</span></span><br><span class="line">xsync /home/admin/bin/</span><br><span class="line"><span class="meta">#</span><span class="bash"> hadoop</span></span><br><span class="line">xsync /opt/module/hadoop-3.1.3/</span><br><span class="line"><span class="meta">#</span><span class="bash"> my_env.sh</span></span><br><span class="line">xsync /etc/profile.d/my_env.sh</span><br></pre></td></tr></table></figure>

<h4 id="启动集群测试"><a href="#启动集群测试" class="headerlink" title="启动集群测试"></a>启动集群测试</h4><figure class="highlight shell"><table><tr><td class="code"><pre><span class="line"><span class="meta">#</span><span class="bash"> 启动</span></span><br><span class="line">myhadoop.sh start</span><br><span class="line"><span class="meta">#</span><span class="bash"> jps</span></span><br><span class="line">jpsall</span><br></pre></td></tr></table></figure>



<h2 id="Hive-搭建"><a href="#Hive-搭建" class="headerlink" title="Hive 搭建"></a>Hive 搭建</h2><p>hive 的元素据选择存储在mysql中，因此需要安装 mysql</p>
<h4 id="hive-包准备"><a href="#hive-包准备" class="headerlink" title="hive 包准备"></a>hive 包准备</h4><p>apache-hive-3.0.0-bin.tar.gz (从硬盘中获取该文件，上传到 hadoop102/opt/software/目录下)</p>
<p>解压到 /opt/module/ 目录下，重命名为 hive-3.0.0</p>
<h4 id="MySQL安装"><a href="#MySQL安装" class="headerlink" title="MySQL安装"></a>MySQL安装</h4><p>参考文章：<a href="https://hello-github-ui.github.io/posts/8567/">https://hello-github-ui.github.io/posts/8567/</a></p>
<h4 id="配置hive文件"><a href="#配置hive文件" class="headerlink" title="配置hive文件"></a>配置hive文件</h4><p>修改/opt/module/hive-3.0.0/conf/hive-env.sh</p>
<p><code>cp hive-env.sh.template hive-env.sh</code></p>
<p>增加内容如下：</p>
<figure class="highlight shell"><table><tr><td class="code"><pre><span class="line">export HADOOP_HOME=/opt/module/hadoop-3.1.3</span><br><span class="line">export HIVE_CONF_DIR=/opt/module/hive-3.0.0/conf</span><br><span class="line">export HIVE_AUX_JARS_PATH=/opt/module/hive-3.0.0/lib</span><br></pre></td></tr></table></figure>

<h4 id="配置hive-site-xml文件"><a href="#配置hive-site-xml文件" class="headerlink" title="配置hive-site.xml文件"></a>配置hive-site.xml文件</h4><p>直接新建 hive-site.xml 文件</p>
<p>vim hive-site.xml</p>
<figure class="highlight xml"><table><tr><td class="code"><pre><span class="line"><span class="meta">&lt;?xml version=&quot;1.0&quot; encoding=&quot;UTF-8&quot; standalone=&quot;no&quot;?&gt;</span></span><br><span class="line"><span class="meta">&lt;?xml-stylesheet type=&quot;text/xsl&quot; href=&quot;configuration.xsl&quot;?&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">configuration</span>&gt;</span></span><br><span class="line">        <span class="comment">&lt;!--不使用本地数据库 --&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">                <span class="tag">&lt;<span class="name">name</span>&gt;</span>hive.metastore.local<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">                <span class="tag">&lt;<span class="name">value</span>&gt;</span>false<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line"></span><br><span class="line">        <span class="comment">&lt;!--Default数据仓库原始位置是在hdfs上：/user/hive/warehouse路径下--&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">                <span class="tag">&lt;<span class="name">name</span>&gt;</span>hive.metastore.warehouse.dir<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">                <span class="tag">&lt;<span class="name">value</span>&gt;</span>/hive/warehouse<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line">                <span class="comment">&lt;!--先在HDFS创建目录--&gt;</span></span><br><span class="line">                <span class="tag">&lt;<span class="name">description</span>&gt;</span>location of default database for the warehouse<span class="tag">&lt;/<span class="name">description</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line"></span><br><span class="line">        <span class="comment">&lt;!--Hive用来存储不同阶段的MapReduce的执行计划的目录，同时也存储中间输出结果--&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">                <span class="tag">&lt;<span class="name">name</span>&gt;</span>hive.exec.scratchdir<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">                <span class="tag">&lt;<span class="name">value</span>&gt;</span>/tmp/hive<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line">                <span class="comment">&lt;!--先在HDFS创建目录--&gt;</span></span><br><span class="line">        <span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line"></span><br><span class="line">        <span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">                <span class="tag">&lt;<span class="name">name</span>&gt;</span>hive.scratch.dir.permission<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">                <span class="tag">&lt;<span class="name">value</span>&gt;</span>777<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line">                <span class="tag">&lt;<span class="name">description</span>&gt;</span>The permission for the user specific scratch directories that get created.<span class="tag">&lt;/<span class="name">description</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line"></span><br><span class="line">        <span class="comment">&lt;!--当Hive运行在本地模式时配置--&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">                <span class="tag">&lt;<span class="name">name</span>&gt;</span>hive.exec.local.scratchdir<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">                <span class="tag">&lt;<span class="name">value</span>&gt;</span>/opt/module/hive-3.0.0/tmp/hive/root<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line">                <span class="comment">&lt;!--先在本地创建目录--&gt;</span></span><br><span class="line">                <span class="tag">&lt;<span class="name">description</span>&gt;</span>Local scratch space for Hive jobs<span class="tag">&lt;/<span class="name">description</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line"></span><br><span class="line">        <span class="comment">&lt;!--远程资源下载的临时目录--&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">                <span class="tag">&lt;<span class="name">name</span>&gt;</span>hive.downloaded.resources.dir<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">                <span class="tag">&lt;<span class="name">value</span>&gt;</span>/opt/module/hive-3.0.0/tmp/resources<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line">                <span class="comment">&lt;!--先在本地创建目录--&gt;</span></span><br><span class="line">                <span class="tag">&lt;<span class="name">description</span>&gt;</span>Temporary local directory for added resources in the remote file system.<span class="tag">&lt;/<span class="name">description</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line"></span><br><span class="line">        <span class="comment">&lt;!--配置Metastore到MySql--&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">                <span class="tag">&lt;<span class="name">name</span>&gt;</span>javax.jdo.option.ConnectionURL<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">                <span class="tag">&lt;<span class="name">value</span>&gt;</span>jdbc:mysql://localhost:3306/metastore?createDatabaseIfNotExist=true<span class="symbol">&amp;amp;</span>useSSL=false<span class="symbol">&amp;amp;</span>allowPublicKeyRetrieval=true<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line">                <span class="comment">&lt;!--XML需要转义，&amp;转义为&amp;amp;--&gt;</span></span><br><span class="line">                <span class="tag">&lt;<span class="name">description</span>&gt;</span>JDBC connect string for a JDBC metastore<span class="tag">&lt;/<span class="name">description</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line"></span><br><span class="line">        <span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">                <span class="tag">&lt;<span class="name">name</span>&gt;</span>javax.jdo.option.ConnectionDriverName<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">                <span class="tag">&lt;<span class="name">value</span>&gt;</span>com.mysql.cj.jdbc.Driver<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line">                <span class="tag">&lt;<span class="name">description</span>&gt;</span>Driver class name for a JDBC metastore<span class="tag">&lt;/<span class="name">description</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line"></span><br><span class="line">        <span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">                <span class="tag">&lt;<span class="name">name</span>&gt;</span>javax.jdo.option.ConnectionUserName<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">                <span class="tag">&lt;<span class="name">value</span>&gt;</span>root<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line">                <span class="tag">&lt;<span class="name">description</span>&gt;</span>username to use against metastore database<span class="tag">&lt;/<span class="name">description</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line"></span><br><span class="line">        <span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">                <span class="tag">&lt;<span class="name">name</span>&gt;</span>javax.jdo.option.ConnectionPassword<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">                <span class="tag">&lt;<span class="name">value</span>&gt;</span>123456<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line">                <span class="tag">&lt;<span class="name">description</span>&gt;</span>password to use against metastore database<span class="tag">&lt;/<span class="name">description</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line"></span><br><span class="line">        <span class="comment">&lt;!--查询后显示当前数据库，以及查询表的头信息配置--&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">                <span class="tag">&lt;<span class="name">name</span>&gt;</span>hive.cli.print.header<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">                <span class="tag">&lt;<span class="name">value</span>&gt;</span>true<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line"></span><br><span class="line">        <span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">                <span class="tag">&lt;<span class="name">name</span>&gt;</span>hive.cli.print.current.db<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">                <span class="tag">&lt;<span class="name">value</span>&gt;</span>true<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line"></span><br><span class="line">        <span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">                <span class="tag">&lt;<span class="name">name</span>&gt;</span>hive.metastore.schema.verification<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">                <span class="tag">&lt;<span class="name">value</span>&gt;</span>false<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line"></span><br><span class="line">        <span class="comment">&lt;!--metastore的端口--&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">                <span class="tag">&lt;<span class="name">name</span>&gt;</span>hive.metastore.uris<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">                <span class="tag">&lt;<span class="name">value</span>&gt;</span>thrift://hadoop102:9083<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line"></span><br><span class="line"><span class="tag">&lt;/<span class="name">configuration</span>&gt;</span></span><br></pre></td></tr></table></figure>

<h4 id="hive连接MySQL驱动连接包"><a href="#hive连接MySQL驱动连接包" class="headerlink" title="hive连接MySQL驱动连接包"></a>hive连接MySQL驱动连接包</h4><p>拷贝 mysql-connector-java-8.0.26.jar 包 ，将其放到 /opt/module/hive-3.0.0/lib 目录下。</p>
<p>硬盘上mysql目录下应该有这个 connector 包</p>
<h4 id="配置-hive-日志"><a href="#配置-hive-日志" class="headerlink" title="配置 hive 日志"></a>配置 hive 日志</h4><figure class="highlight shell"><table><tr><td class="code"><pre><span class="line">mv hive-log4j2.properties.template hive-log4j2.properties</span><br></pre></td></tr></table></figure>

<p>在 /opt/module/hive-3.0.0/ 新建目录 log，用于存放 hive 日志</p>
<p>修改 hive-log4j2.properties 配置：</p>
<figure class="highlight shell"><table><tr><td class="code"><pre><span class="line"><span class="meta">#</span><span class="bash"> 找到改行内容，将其修改为如下</span></span><br><span class="line">property.hive.log.dir = /opt/module/hive-3.0.0/log</span><br></pre></td></tr></table></figure>

<h4 id="新建hive仓库目录"><a href="#新建hive仓库目录" class="headerlink" title="新建hive仓库目录"></a>新建hive仓库目录</h4><figure class="highlight shell"><table><tr><td class="code"><pre><span class="line">hdfs dfs -mkdir -p /hive/warehouse</span><br><span class="line">hdfs dfs -mkdir -p /tmp/hive</span><br></pre></td></tr></table></figure>

<h4 id="配置环境变量"><a href="#配置环境变量" class="headerlink" title="配置环境变量"></a>配置环境变量</h4><figure class="highlight shell"><table><tr><td class="code"><pre><span class="line"><span class="meta">#</span><span class="bash"> vim /etc/profile.d/my_env.sh</span></span><br><span class="line"><span class="meta">#</span><span class="bash">JAVA_HOME</span></span><br><span class="line">export JAVA_HOME=/opt/module/jdk1.8.0_311</span><br><span class="line">export PATH=$PATH:$JAVA_HOME/bin</span><br><span class="line"><span class="meta"></span></span><br><span class="line"><span class="meta">#</span><span class="bash">HADOOP_HOME</span></span><br><span class="line">export HADOOP_HOME=/opt/module/hadoop-3.1.3</span><br><span class="line">export PATH=$PATH:$HADOOP_HOME/bin</span><br><span class="line">export PATH=$PATH:$HADOOP_HOME/sbin</span><br><span class="line"><span class="meta"></span></span><br><span class="line"><span class="meta">#</span><span class="bash">HIVE_HOME</span></span><br><span class="line">export HIVE_HOME=/opt/module/hive-3.0.0</span><br><span class="line">export PATH=$PATH:$HIVE_HOME/bin</span><br><span class="line"></span><br><span class="line">export HADOOP_COMMON_LIB_NATIVE_DIR=$HADOOP_HOME/lib/native</span><br><span class="line">export HADOOP_OPTS=-Djava.library.path=$HADOOP_HOME/lib</span><br></pre></td></tr></table></figure>

<h4 id="元数据库初始化"><a href="#元数据库初始化" class="headerlink" title="元数据库初始化"></a>元数据库初始化</h4><p>进入目录 /opt/module/hive-3.0.0/bin</p>
<figure class="highlight shell"><table><tr><td class="code"><pre><span class="line">schematool -dbType mysql -initSchema</span><br></pre></td></tr></table></figure>

<h4 id="启动hive"><a href="#启动hive" class="headerlink" title="启动hive"></a>启动hive</h4><blockquote>
<p>注意：启动hive前，一定要先启动 metastore 或者 hiveserver2<br>Spark Thrift Server 是 Spark 社区基于 HiveServer2 实现的一个 Thrift 服务。旨在无缝兼容HiveServer2。因为 Spark Thrift Server 的接口和协议都和 HiveServer2 完全一致，因此我们部<br>署好 Spark Thrift Server 后，可以直接使用 hive 的 beeline 访问 Spark Thrift Server 执行相关语句。Spark Thrift Server 的目的也只是取代 HiveServer2，因此它依旧可以和 Hive Metastore<br>进行交互，获取到 hive 的元数据。</p>
</blockquote>
<p>此处通过 Thrift 服务来启动 客户端配置：<code>sh /opt/module/spark-yarn/sbin/start-thriftserver.sh</code><br>使用 beeline 连接 Thrift Server: <code>bin/beeline -u jdbc:hive2://hadoop102:10000 -n admin</code></p>
<blockquote>
<p>当然你也可以通过启动 metasotre，进入hive的bin目录 : <code>./hive --service metastore &amp;</code>，然后再启动hive: <code>hive</code></p>
</blockquote>
<h2 id="Spark-搭建"><a href="#Spark-搭建" class="headerlink" title="Spark 搭建"></a>Spark 搭建</h2><p>spark-3.0.0-bin-hadoop3.2.tgz 准备，从硬盘上获取</p>
<p>上传到 hadoop102  /opt/software/ 上，解压缩到 /opt/module/ 下</p>
<p>重命名为 mv spark-3.0.0-bin-hadoop3.2 spark-yarn</p>
<p>编辑文件 vim /opt/module/spark-yarn/conf/spark-env.sh 文末增加如下内容</p>
<figure class="highlight shell"><table><tr><td class="code"><pre><span class="line">export JAVA_HOME=/opt/module/jdk1.8.0_311</span><br><span class="line">YARN_CONF_DIR=/opt/module/hadoop-3.1.3/etc/hadoop</span><br><span class="line"></span><br><span class="line">export SPARK_HISTORY_OPTS=&quot;</span><br><span class="line"> -Dspark.history.ui.port=18080</span><br><span class="line"> -Dspark.history.fs.logDirectory=hdfs://hadoop102:8020/directory</span><br><span class="line"> -Dspark.history.retainedApplications=30</span><br></pre></td></tr></table></figure>

<h4 id="Spark连接外置hive"><a href="#Spark连接外置hive" class="headerlink" title="Spark连接外置hive"></a>Spark连接外置hive</h4><p>通常来说如果你下载的是二进制版本的Spark，它应该已经在编译时添加了hive支持</p>
<p>若要把 Spark SQL连接到一个部署好的hive上，你必须把 hive-site.xml 复制到 Spark的配置文件目录中，即使没有部署好 hive，Spark SQL 也可以运行。因为实际中这种方式比较少，所以我们这里主要来讲 连接 外置hive</p>
<p>如果想连接外部已经部署好的 Hive，需要通过以下几个步骤：</p>
<p>➢ Spark 要接管 Hive 需要把 <code>/opt/module/hive-3.0.0/conf/hive-site.xml</code> 拷贝到 <code>/opt/module/spark-yarn/conf/</code> 目录下</p>
<p>➢ 把 Mysql 的驱动 copy 到 <code>/opt/module/spark-yarn/jars/</code>目录下</p>
<p>➢ 如果访问不到 hdfs，则需要把 <code>/opt/module/hadoop-3.1.3/etc/hadoop/core-site.xml</code> 和 <code>/opt/module/hadoop-3.1.3/etc/hadoop/hdfs-site.xml</code> 拷贝到 conf/目录下</p>
<p>➢ 重启 spark-shell</p>
</article><div class="post-copyright"><div class="post-copyright__author"><span class="post-copyright-meta">Author: </span><span class="post-copyright-info"><a href="mailto:undefined">Lorem Moon</a></span></div><div class="post-copyright__type"><span class="post-copyright-meta">Link: </span><span class="post-copyright-info"><a href="https://hello-github-ui.github.io/posts/1553/">https://hello-github-ui.github.io/posts/1553/</a></span></div><div class="post-copyright__notice"><span class="post-copyright-meta">Copyright Notice: </span><span class="post-copyright-info">All articles in this blog are licensed under <a target="_blank" rel="noopener" href="https://creativecommons.org/licenses/by-nc-sa/4.0/">CC BY-NC-SA 4.0</a> unless stating additionally.</span></div></div><div class="tag_share"><div class="post-meta__tag-list"><a class="post-meta__tags" href="/tags/Hadoop/">Hadoop</a><a class="post-meta__tags" href="/tags/BigData/">BigData</a><a class="post-meta__tags" href="/tags/Spark/">Spark</a><a class="post-meta__tags" href="/tags/Hive/">Hive</a></div><div class="post_share"><div class="social-share" data-image="https://cdn.jsdelivr.net/gh/jerryc127/CDN@latest/cover/default_bg3.png" data-sites="facebook,twitter,wechat,weibo,qq"></div><link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/social-share.js/dist/css/share.min.css" media="print" onload="this.media='all'"><script src="https://cdn.jsdelivr.net/npm/social-share.js/dist/js/social-share.min.js" defer></script></div></div><hr/><div id="post-comment"><div class="comment-head"><div class="comment-headline"><i class="fas fa-comments fa-fw"></i><span> Comment</span></div></div><div class="comment-wrap"><div><div class="vcomment" id="vcomment"></div></div></div></div></div><div class="aside-content" id="aside-content"><div class="card-widget card-info"><div class="is-center"><div class="avatar-img"><img src="https://pic.imgdb.cn/item/6159d89b2ab3f51d91b0e8d0.jpg" onerror="this.onerror=null;this.src='/img/friend_404.gif'" alt="avatar"/></div><div class="author-info__name">Lorem Moon</div><div class="author-info__description">即已成过往，便随它而去吧。</div></div><div class="card-info-data is-center"><div class="card-info-data-item"><a href="/archives/"><div class="headline">Articles</div><div class="length-num">60</div></a></div><div class="card-info-data-item"><a href="/tags/"><div class="headline">Tags</div><div class="length-num">46</div></a></div><div class="card-info-data-item"><a href="/categories/"><div class="headline">Categories</div><div class="length-num">5</div></a></div></div><a class="button--animated" id="card-info-btn" target="_blank" rel="noopener" href="https://github.com/xxxxxx"><i class="fab fa-github"></i><span>Follow Me</span></a><div class="card-info-social-icons is-center"><a class="social-icon" href="https://github.com/hello-github-ui" target="_blank" title="Github"><i class="fab fa-github"></i></a><a class="social-icon" href="mailto:util.you.come@gmail.com" target="_blank" title="Email"><i class="fas fa-envelope"></i></a></div></div><div class="card-widget card-announcement"><div class="item-headline"><i class="fas fa-bullhorn card-announcement-animation"></i><span>Announcement</span></div><div class="announcement_content">This is my Blog</div></div><div class="sticky_layout"><div class="card-widget" id="card-toc"><div class="item-headline"><i class="fas fa-stream"></i><span>Catalog</span><span class="toc-percentage"></span></div><div class="toc-content"><ol class="toc"><li class="toc-item toc-level-1"><a class="toc-link" href="#%E5%A4%A7%E6%95%B0%E6%8D%AE%E5%9F%BA%E7%A1%80"><span class="toc-number">1.</span> <span class="toc-text">大数据基础</span></a><ol class="toc-child"><li class="toc-item toc-level-2"><a class="toc-link" href="#Hadoop-Yarn%E9%9B%86%E7%BE%A4%E7%9A%84%E6%90%AD%E5%BB%BA"><span class="toc-number">1.1.</span> <span class="toc-text">Hadoop Yarn集群的搭建</span></a><ol class="toc-child"><li class="toc-item toc-level-3"><a class="toc-link" href="#%E8%99%9A%E6%8B%9F%E6%9C%BA%E7%9A%84%E5%87%86%E5%A4%87"><span class="toc-number">1.1.1.</span> <span class="toc-text">虚拟机的准备</span></a><ol class="toc-child"><li class="toc-item toc-level-4"><a class="toc-link" href="#%E4%BF%AE%E6%94%B9-IP"><span class="toc-number">1.1.1.1.</span> <span class="toc-text">修改 IP</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#%E4%BF%AE%E6%94%B9%E4%B8%BB%E6%9C%BA%E5%90%8D"><span class="toc-number">1.1.1.2.</span> <span class="toc-text">修改主机名</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#%E9%85%8D%E7%BD%AE-Linux-%E5%85%8B%E9%9A%86%E6%9C%BA%E4%B8%BB%E6%9C%BA%E5%90%8D%E7%A7%B0%E6%98%A0%E5%B0%84-hosts-%E6%96%87%E4%BB%B6%EF%BC%8C%E6%89%93%E5%BC%80-etc-hosts"><span class="toc-number">1.1.1.3.</span> <span class="toc-text">配置 Linux 克隆机主机名称映射 hosts 文件，打开&#x2F;etc&#x2F;hosts</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#Windows-host%E6%96%87%E4%BB%B6%E9%85%8D%E7%BD%AE"><span class="toc-number">1.1.1.4.</span> <span class="toc-text">Windows host文件配置</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#%E5%8D%B8%E8%BD%BD%E8%87%AA%E5%B8%A6%E7%9A%84-jdk"><span class="toc-number">1.1.1.5.</span> <span class="toc-text">卸载自带的 jdk</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#%E5%85%B3%E9%97%AD%E9%98%B2%E7%81%AB%E5%A2%99%E5%B9%B6%E7%A6%81%E7%94%A8%E5%BC%80%E6%9C%BA%E8%87%AA%E5%90%AF%E5%8A%A8%E9%98%B2%E7%81%AB%E5%A2%99"><span class="toc-number">1.1.1.6.</span> <span class="toc-text">关闭防火墙并禁用开机自启动防火墙</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#%E5%88%9B%E5%BB%BA%E6%99%AE%E9%80%9A%E7%94%A8%E6%88%B7"><span class="toc-number">1.1.1.7.</span> <span class="toc-text">创建普通用户</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#%E9%85%8D%E7%BD%AE-admin-%E7%94%A8%E6%88%B7%E5%85%B7%E6%9C%89-root-%E6%9D%83%E9%99%90%EF%BC%8C%E6%96%B9%E4%BE%BF%E5%90%8E%E6%9C%9F%E5%8A%A0-sudo-%E6%89%A7%E8%A1%8C-root-%E6%9D%83%E9%99%90%E7%9A%84%E5%91%BD%E4%BB%A4"><span class="toc-number">1.1.1.8.</span> <span class="toc-text">配置 admin 用户具有 root 权限，方便后期加 sudo 执行 root 权限的命令</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#%E5%9C%A8-opt-%E7%9B%AE%E5%BD%95%E4%B8%8B%E5%88%9B%E5%BB%BA%E6%96%87%E4%BB%B6%E5%A4%B9%EF%BC%8C%E5%B9%B6%E4%BF%AE%E6%94%B9%E6%89%80%E5%B1%9E%E4%B8%BB%E5%92%8C%E6%89%80%E5%B1%9E%E7%BB%84"><span class="toc-number">1.1.1.9.</span> <span class="toc-text">在&#x2F;opt 目录下创建文件夹，并修改所属主和所属组</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#Hadoop102%E4%B8%8A%E5%AE%89%E8%A3%85jdk"><span class="toc-number">1.1.1.10.</span> <span class="toc-text">Hadoop102上安装jdk</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#%E7%BC%96%E5%86%99%E9%9B%86%E7%BE%A4%E5%88%86%E5%8F%91%E8%84%9A%E6%9C%AC-xsync"><span class="toc-number">1.1.1.11.</span> <span class="toc-text">编写集群分发脚本 xsync</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#SSH-%E6%97%A0%E5%AF%86%E7%99%BB%E5%BD%95%E9%85%8D%E7%BD%AE"><span class="toc-number">1.1.1.12.</span> <span class="toc-text">SSH 无密登录配置</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#%E5%90%8C%E6%AD%A5-jdk-%E5%88%B0-Hadoop103%EF%BC%8CHadoop104"><span class="toc-number">1.1.1.13.</span> <span class="toc-text">同步 jdk 到 Hadoop103，Hadoop104</span></a></li></ol></li><li class="toc-item toc-level-3"><a class="toc-link" href="#Hadoop-%E5%AE%89%E8%A3%85"><span class="toc-number">1.1.2.</span> <span class="toc-text">Hadoop 安装</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#Hadoop-Yarn%E9%9B%86%E7%BE%A4%E9%85%8D%E7%BD%AE"><span class="toc-number">1.1.3.</span> <span class="toc-text">Hadoop Yarn集群配置</span></a><ol class="toc-child"><li class="toc-item toc-level-4"><a class="toc-link" href="#%E9%85%8D%E7%BD%AE-core-site-xml"><span class="toc-number">1.1.3.1.</span> <span class="toc-text">配置 core-site.xml</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#%E9%85%8D%E7%BD%AE-hdfs-site-xml"><span class="toc-number">1.1.3.2.</span> <span class="toc-text">配置 hdfs-site.xml</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#%E9%85%8D%E7%BD%AE-yarn-site-xml"><span class="toc-number">1.1.3.3.</span> <span class="toc-text">配置 yarn-site.xml</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#%E9%85%8D%E7%BD%AE-mapred-site-xml"><span class="toc-number">1.1.3.4.</span> <span class="toc-text">配置 mapred-site.xml</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#%E9%85%8D%E7%BD%AE-workers"><span class="toc-number">1.1.3.5.</span> <span class="toc-text">配置 workers</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#%E9%85%8D%E7%BD%AE%E5%8E%86%E5%8F%B2%E6%9C%8D%E5%8A%A1%E5%99%A8"><span class="toc-number">1.1.3.6.</span> <span class="toc-text">配置历史服务器</span></a><ol class="toc-child"><li class="toc-item toc-level-5"><a class="toc-link" href="#%E9%85%8D%E7%BD%AE-mapred-site-xml-1"><span class="toc-number">1.1.3.6.1.</span> <span class="toc-text">配置 mapred-site.xml</span></a></li></ol></li><li class="toc-item toc-level-4"><a class="toc-link" href="#%E9%85%8D%E7%BD%AE%E6%97%A5%E5%BF%97%E7%9A%84%E8%81%9A%E9%9B%86"><span class="toc-number">1.1.3.7.</span> <span class="toc-text">配置日志的聚集</span></a><ol class="toc-child"><li class="toc-item toc-level-5"><a class="toc-link" href="#%E9%85%8D%E7%BD%AE-yarn-site-xml-1"><span class="toc-number">1.1.3.7.1.</span> <span class="toc-text">配置 yarn-site.xml</span></a></li></ol></li><li class="toc-item toc-level-4"><a class="toc-link" href="#%E9%9B%86%E7%BE%A4%E5%90%AF%E5%8A%A8-%E5%81%9C%E6%AD%A2%E6%96%B9%E5%BC%8F"><span class="toc-number">1.1.3.8.</span> <span class="toc-text">集群启动&#x2F;停止方式</span></a><ol class="toc-child"><li class="toc-item toc-level-5"><a class="toc-link" href="#Hadoop-%E9%9B%86%E7%BE%A4%E5%90%AF%E5%81%9C%E8%84%9A%E6%9C%AC%EF%BC%88%E5%8C%85%E5%90%AB-HDFS%EF%BC%8CYarn%EF%BC%8CHistoryserver%EF%BC%89%EF%BC%9Amyhadoop-sh"><span class="toc-number">1.1.3.8.1.</span> <span class="toc-text">Hadoop 集群启停脚本（包含 HDFS，Yarn，Historyserver）：myhadoop.sh</span></a></li></ol></li><li class="toc-item toc-level-4"><a class="toc-link" href="#%E6%9F%A5%E7%9C%8B%E4%B8%89%E5%8F%B0%E6%9C%8D%E5%8A%A1%E5%99%A8-Java-%E8%BF%9B%E7%A8%8B%E8%84%9A%E6%9C%AC%EF%BC%9Ajpsall"><span class="toc-number">1.1.3.9.</span> <span class="toc-text">查看三台服务器 Java 进程脚本：jpsall</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#%E5%88%86%E5%8F%91%E5%86%85%E5%AE%B9%E5%88%B0%E5%85%B6%E5%AE%83%E4%B8%BB%E6%9C%BA"><span class="toc-number">1.1.3.10.</span> <span class="toc-text">分发内容到其它主机</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#%E5%90%AF%E5%8A%A8%E9%9B%86%E7%BE%A4%E6%B5%8B%E8%AF%95"><span class="toc-number">1.1.3.11.</span> <span class="toc-text">启动集群测试</span></a></li></ol></li></ol></li><li class="toc-item toc-level-2"><a class="toc-link" href="#Hive-%E6%90%AD%E5%BB%BA"><span class="toc-number">1.2.</span> <span class="toc-text">Hive 搭建</span></a><ol class="toc-child"><li class="toc-item toc-level-4"><a class="toc-link" href="#hive-%E5%8C%85%E5%87%86%E5%A4%87"><span class="toc-number">1.2.0.1.</span> <span class="toc-text">hive 包准备</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#MySQL%E5%AE%89%E8%A3%85"><span class="toc-number">1.2.0.2.</span> <span class="toc-text">MySQL安装</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#%E9%85%8D%E7%BD%AEhive%E6%96%87%E4%BB%B6"><span class="toc-number">1.2.0.3.</span> <span class="toc-text">配置hive文件</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#%E9%85%8D%E7%BD%AEhive-site-xml%E6%96%87%E4%BB%B6"><span class="toc-number">1.2.0.4.</span> <span class="toc-text">配置hive-site.xml文件</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#hive%E8%BF%9E%E6%8E%A5MySQL%E9%A9%B1%E5%8A%A8%E8%BF%9E%E6%8E%A5%E5%8C%85"><span class="toc-number">1.2.0.5.</span> <span class="toc-text">hive连接MySQL驱动连接包</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#%E9%85%8D%E7%BD%AE-hive-%E6%97%A5%E5%BF%97"><span class="toc-number">1.2.0.6.</span> <span class="toc-text">配置 hive 日志</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#%E6%96%B0%E5%BB%BAhive%E4%BB%93%E5%BA%93%E7%9B%AE%E5%BD%95"><span class="toc-number">1.2.0.7.</span> <span class="toc-text">新建hive仓库目录</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#%E9%85%8D%E7%BD%AE%E7%8E%AF%E5%A2%83%E5%8F%98%E9%87%8F"><span class="toc-number">1.2.0.8.</span> <span class="toc-text">配置环境变量</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#%E5%85%83%E6%95%B0%E6%8D%AE%E5%BA%93%E5%88%9D%E5%A7%8B%E5%8C%96"><span class="toc-number">1.2.0.9.</span> <span class="toc-text">元数据库初始化</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#%E5%90%AF%E5%8A%A8hive"><span class="toc-number">1.2.0.10.</span> <span class="toc-text">启动hive</span></a></li></ol></li></ol></li><li class="toc-item toc-level-2"><a class="toc-link" href="#Spark-%E6%90%AD%E5%BB%BA"><span class="toc-number">1.3.</span> <span class="toc-text">Spark 搭建</span></a><ol class="toc-child"><li class="toc-item toc-level-4"><a class="toc-link" href="#Spark%E8%BF%9E%E6%8E%A5%E5%A4%96%E7%BD%AEhive"><span class="toc-number">1.3.0.1.</span> <span class="toc-text">Spark连接外置hive</span></a></li></ol></li></ol></li></ol></li></ol></div></div><div class="card-widget card-recent-post"><div class="item-headline"><i class="fas fa-history"></i><span>Recent Post</span></div><div class="aside-list"><div class="aside-list-item"><a class="thumbnail" href="/posts/4723/" title="Redis主从搭建"><img src="https://pic.imgdb.cn/item/61210fda4907e2d39c439d0c.jpg" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="Redis主从搭建"/></a><div class="content"><a class="title" href="/posts/4723/" title="Redis主从搭建">Redis主从搭建</a><time datetime="2022-03-08T12:34:42.574Z" title="Created 2022-03-08 20:34:42">2022-03-08</time></div></div><div class="aside-list-item"><a class="thumbnail" href="/posts/43818/" title="Win10开启上帝模式"><img src="https://pic.imgdb.cn/item/621ddc1c2ab3f51d914c66cb.png" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="Win10开启上帝模式"/></a><div class="content"><a class="title" href="/posts/43818/" title="Win10开启上帝模式">Win10开启上帝模式</a><time datetime="2022-03-08T12:24:42.000Z" title="Created 2022-03-08 20:24:42">2022-03-08</time></div></div><div class="aside-list-item"><a class="thumbnail" href="/posts/55966/" title="Ubuntu修改静态IP"><img src="https://pic.imgdb.cn/item/621ddc1c2ab3f51d914c66cb.png" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="Ubuntu修改静态IP"/></a><div class="content"><a class="title" href="/posts/55966/" title="Ubuntu修改静态IP">Ubuntu修改静态IP</a><time datetime="2022-03-04T16:11:25.000Z" title="Created 2022-03-05 00:11:25">2022-03-05</time></div></div><div class="aside-list-item"><a class="thumbnail" href="/posts/42154/" title="Java封装同步锁"><img src="https://pic.imgdb.cn/item/621ddc1c2ab3f51d914c66cb.png" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="Java封装同步锁"/></a><div class="content"><a class="title" href="/posts/42154/" title="Java封装同步锁">Java封装同步锁</a><time datetime="2022-03-02T12:50:01.000Z" title="Created 2022-03-02 20:50:01">2022-03-02</time></div></div><div class="aside-list-item"><a class="thumbnail" href="/posts/1553/" title="初识大数据(一)"><img src="https://cdn.jsdelivr.net/gh/jerryc127/CDN@latest/cover/default_bg3.png" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="初识大数据(一)"/></a><div class="content"><a class="title" href="/posts/1553/" title="初识大数据(一)">初识大数据(一)</a><time datetime="2022-03-01T07:53:43.000Z" title="Created 2022-03-01 15:53:43">2022-03-01</time></div></div></div></div></div></div></main><footer id="footer" style="background-image: url('https://cdn.jsdelivr.net/gh/jerryc127/CDN@latest/cover/default_bg3.png')"><div id="footer-wrap"><div class="copyright">&copy;2017 - 2022 By Lorem Moon</div><div class="framework-info"><span>Framework </span><a target="_blank" rel="noopener" href="https://hexo.io">Hexo</a><span class="footer-separator">|</span><span>Theme </span><a target="_blank" rel="noopener" href="https://github.com/jerryc127/hexo-theme-butterfly">Butterfly</a></div><div class="footer_custom_text">Hi, welcome to my <a target="_blank" rel="noopener" href="https://hello-blogger-ui.blogspot.com/">google blog</a>!</div></div></footer></div><div id="rightside"><div id="rightside-config-hide"><button id="readmode" type="button" title="Read Mode"><i class="fas fa-book-open"></i></button><button id="darkmode" type="button" title="Switch Between Light And Dark Mode"><i class="fas fa-adjust"></i></button><button id="hide-aside-btn" type="button" title="Toggle between single-column and double-column"><i class="fas fa-arrows-alt-h"></i></button></div><div id="rightside-config-show"><button id="rightside_config" type="button" title="Setting"><i class="fas fa-cog fa-spin"></i></button><button class="close" id="mobile-toc-button" type="button" title="Table Of Contents"><i class="fas fa-list-ul"></i></button><a id="to_comment" href="#post-comment" title="Scroll To Comments"><i class="fas fa-comments"></i></a><button id="go-up" type="button" title="Back To Top"><i class="fas fa-arrow-up"></i></button></div></div><div id="algolia-search"><div class="search-dialog"><nav class="search-nav"><span class="search-dialog-title">Algolia</span><button class="search-close-button"><i class="fas fa-times"></i></button></nav><div class="search-wrap"><div id="algolia-search-input"></div><hr/><div id="algolia-search-results"><div id="algolia-hits"></div><div id="algolia-pagination"></div><div id="algolia-stats"></div></div></div></div><div id="search-mask"></div></div><div><script src="/js/utils.js"></script><script src="/js/main.js"></script><script src="https://cdn.jsdelivr.net/npm/@fancyapps/ui/dist/fancybox.umd.js"></script><script src="https://cdn.jsdelivr.net/npm/algoliasearch@4/dist/algoliasearch-lite.umd.js"></script><script src="https://cdn.jsdelivr.net/npm/instantsearch.js@4/dist/instantsearch.production.min.js"></script><script src="/js/search/algolia.js"></script><div class="js-pjax"><script>function loadValine () {
  function initValine () {
    const valine = new Valine(Object.assign({
      el: '#vcomment',
      appId: 'mFM6i90bN6KqyTH6tmC3q7dz-gzGzoHsz',
      appKey: 'T2tsKoiKjMaB0wgtIAgSz6Ul',
      avatar: 'monsterid',
      serverURLs: '',
      emojiMaps: "",
      path: window.location.pathname,
      visitor: false
    }, null))
  }

  if (typeof Valine === 'function') initValine() 
  else getScript('https://cdn.jsdelivr.net/npm/valine/dist/Valine.min.js').then(initValine)
}

if ('Valine' === 'Valine' || !false) {
  if (false) btf.loadComment(document.getElementById('vcomment'),loadValine)
  else setTimeout(loadValine, 0)
} else {
  function loadOtherComment () {
    loadValine()
  }
}</script></div><script async data-pjax src="//busuanzi.ibruce.info/busuanzi/2.3/busuanzi.pure.mini.js"></script></div></body></html>